/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-3-25
 */
package code.extract.training;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;

public class KeywordList {
	public static final String COMMENT_SYMBOL = "#";
	private ArrayList<String> content = new ArrayList<String>();
	private ArrayList<Boolean> iscrawledList = new ArrayList<Boolean>();
	private HashSet<String> crawledSet = new HashSet<String>();
	
	public KeywordList(String list_filename, String crawled_filename, String encode){
		initCrawledSet(crawled_filename, encode);
		initKeywordList(list_filename, encode);
	}
	public int initCrawledSet(String filename, String encode){
		int result = 0;
		try{
			File file = new File(filename);
			FileInputStream fis = new FileInputStream(file);
			InputStreamReader isr = null;
			if (encode == null){
				isr = new InputStreamReader(fis);
			} else{
				isr = new InputStreamReader(fis, encode);
			}
			BufferedReader br = new BufferedReader(isr);
			crawledSet.clear();
			String line = br.readLine();
			while (line != null){
				crawledSet.add(line.trim());
				line = br.readLine();
			}
			br.close();
			isr.close();
			fis.close();
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}

	public int initKeywordList(String filename, String encode){
		int result = 0;
		try{
			File file = new File(filename);
			FileInputStream fis = new FileInputStream(file);
			InputStreamReader isr = null;
			if (encode == null){
				isr = new InputStreamReader(fis);
			} else{
				isr = new InputStreamReader(fis, encode);
			}
			BufferedReader br = new BufferedReader(isr);
			content.clear();
			String line = br.readLine();
			while (line != null){
				content.add(line.trim());
				if (line.startsWith("#") || crawledSet.contains(line)){
					iscrawledList.add(true);
				}else{
					iscrawledList.add(false);
				}
				line = br.readLine();
			}
			br.close();
			isr.close();
			fis.close();
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}
	
	public void setContent(ArrayList<String> content) {
		this.content = content;
	}

	public ArrayList<String> getContent() {
		return content;
	}

	public void setIscrawledList(ArrayList<Boolean> iscrawledList) {
		this.iscrawledList = iscrawledList;
	}

	public ArrayList<Boolean> getIscrawledList() {
		return iscrawledList;
	}
	public ArrayList<String> getUncrawledContent(){
		ArrayList<String> result = new ArrayList<String>();
		for (int i=0; i<content.size(); ++i){
			if (iscrawledList.get(i).booleanValue() == false){
				result.add(content.get(i));
			}
		}
		return result;
	}
}
